{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 7.7 Transformerの学習・推論、判定根拠の可視化を実装\n",
    "\n",
    "- 本ファイルでは、ここまでで作成したTransformerモデルとIMDbのDataLoaderを使用してクラス分類を学習させます。テストデータで推論をし、さらに判断根拠となるAttentionを可視化します"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 7.7 学習目標\n",
    "\n",
    "1.\tTransformerの学習を実装できるようになる\n",
    "2.\tTransformerの判定時のAttention可視化を実装できるようになる\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 事前準備\n",
    "\n",
    "- フォルダ「utils」内の関数やクラスを使用します"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# パッケージのimport\n",
    "import numpy as np\n",
    "import random\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "import torchtext\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 乱数のシードを設定\n",
    "torch.manual_seed(1234)\n",
    "np.random.seed(1234)\n",
    "random.seed(1234)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DatasetとDataLoaderを作成"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.dataloader import get_IMDb_DataLoaders_and_TEXT\n",
    "\n",
    "# 読み込み\n",
    "train_dl, val_dl, test_dl, TEXT = get_IMDb_DataLoaders_and_TEXT(\n",
    "    max_length=256, batch_size=64)\n",
    "\n",
    "# 辞書オブジェクトにまとめる\n",
    "dataloaders_dict = {\"train\": train_dl, \"val\": val_dl}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ネットワークモデルの作成"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ネットワーク設定完了\n"
     ]
    }
   ],
   "source": [
    "from utils.transformer import TransformerClassification\n",
    "\n",
    "# モデル構築\n",
    "net = TransformerClassification(\n",
    "    text_embedding_vectors=TEXT.vocab.vectors, d_model=300, max_seq_len=256, output_dim=2)\n",
    "\n",
    "# ネットワークの初期化を定義\n",
    "\n",
    "\n",
    "def weights_init(m):\n",
    "    classname = m.__class__.__name__\n",
    "    if classname.find('Linear') != -1:\n",
    "        # Liner層の初期化\n",
    "        nn.init.kaiming_normal_(m.weight)\n",
    "        if m.bias is not None:\n",
    "            nn.init.constant_(m.bias, 0.0)\n",
    "\n",
    "\n",
    "# 訓練モードに設定\n",
    "net.train()\n",
    "\n",
    "# TransformerBlockモジュールを初期化実行\n",
    "net.net3_1.apply(weights_init)\n",
    "net.net3_2.apply(weights_init)\n",
    "\n",
    "\n",
    "print('ネットワーク設定完了')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 損失関数と最適化手法を定義"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 損失関数の設定\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "# nn.LogSoftmax()を計算してからnn.NLLLoss(negative log likelihood loss)を計算\n",
    "\n",
    "# 最適化手法の設定\n",
    "learning_rate = 2e-5\n",
    "optimizer = optim.Adam(net.parameters(), lr=learning_rate)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 学習・検証を実施"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# モデルを学習させる関数を作成\n",
    "\n",
    "\n",
    "def train_model(net, dataloaders_dict, criterion, optimizer, num_epochs):\n",
    "\n",
    "    # GPUが使えるかを確認\n",
    "    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "    print(\"使用デバイス：\", device)\n",
    "    print('-----start-------')\n",
    "    # ネットワークをGPUへ\n",
    "    net.to(device)\n",
    "\n",
    "    # ネットワークがある程度固定であれば、高速化させる\n",
    "    torch.backends.cudnn.benchmark = True\n",
    "\n",
    "    # epochのループ\n",
    "    for epoch in range(num_epochs):\n",
    "        # epochごとの訓練と検証のループ\n",
    "        for phase in ['train', 'val']:\n",
    "            if phase == 'train':\n",
    "                net.train()  # モデルを訓練モードに\n",
    "            else:\n",
    "                net.eval()   # モデルを検証モードに\n",
    "\n",
    "            epoch_loss = 0.0  # epochの損失和\n",
    "            epoch_corrects = 0  # epochの正解数\n",
    "\n",
    "            # データローダーからミニバッチを取り出すループ\n",
    "            for batch in (dataloaders_dict[phase]):\n",
    "                # batchはTextとLableの辞書オブジェクト\n",
    "\n",
    "                # GPUが使えるならGPUにデータを送る\n",
    "                inputs = batch.Text[0].to(device)  # 文章\n",
    "                labels = batch.Label.to(device)  # ラベル\n",
    "\n",
    "                # optimizerを初期化\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                # 順伝搬（forward）計算\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "\n",
    "                    # mask作成\n",
    "                    input_pad = 1  # 単語のIDにおいて、'<pad>': 1 なので\n",
    "                    input_mask = (inputs != input_pad)\n",
    "\n",
    "                    # Transformerに入力\n",
    "                    outputs, _, _ = net(inputs, input_mask)\n",
    "                    loss = criterion(outputs, labels)  # 損失を計算\n",
    "\n",
    "                    _, preds = torch.max(outputs, 1)  # ラベルを予測\n",
    "\n",
    "                    # 訓練時はバックプロパゲーション\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "\n",
    "                    # 結果の計算\n",
    "                    epoch_loss += loss.item() * inputs.size(0)  # lossの合計を更新\n",
    "                    # 正解数の合計を更新\n",
    "                    epoch_corrects += torch.sum(preds == labels.data)\n",
    "\n",
    "            # epochごとのlossと正解率\n",
    "            epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)\n",
    "            epoch_acc = epoch_corrects.double(\n",
    "            ) / len(dataloaders_dict[phase].dataset)\n",
    "\n",
    "            print('Epoch {}/{} | {:^5} |  Loss: {:.4f} Acc: {:.4f}'.format(epoch+1, num_epochs,\n",
    "                                                                           phase, epoch_loss, epoch_acc))\n",
    "\n",
    "    return net\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "使用デバイス： cuda:0\n",
      "-----start-------\n",
      "Epoch 1/10 | train |  Loss: 0.6039 Acc: 0.6629\n",
      "Epoch 1/10 |  val  |  Loss: 0.4203 Acc: 0.8174\n",
      "Epoch 2/10 | train |  Loss: 0.4382 Acc: 0.8025\n",
      "Epoch 2/10 |  val  |  Loss: 0.3872 Acc: 0.8332\n",
      "Epoch 3/10 | train |  Loss: 0.4130 Acc: 0.8161\n",
      "Epoch 3/10 |  val  |  Loss: 0.3688 Acc: 0.8456\n",
      "Epoch 4/10 | train |  Loss: 0.3862 Acc: 0.8292\n",
      "Epoch 4/10 |  val  |  Loss: 0.3789 Acc: 0.8432\n",
      "Epoch 5/10 | train |  Loss: 0.3718 Acc: 0.8356\n",
      "Epoch 5/10 |  val  |  Loss: 0.3477 Acc: 0.8552\n",
      "Epoch 6/10 | train |  Loss: 0.3601 Acc: 0.8397\n",
      "Epoch 6/10 |  val  |  Loss: 0.3401 Acc: 0.8570\n",
      "Epoch 7/10 | train |  Loss: 0.3515 Acc: 0.8480\n",
      "Epoch 7/10 |  val  |  Loss: 0.3452 Acc: 0.8558\n",
      "Epoch 8/10 | train |  Loss: 0.3435 Acc: 0.8513\n",
      "Epoch 8/10 |  val  |  Loss: 0.3523 Acc: 0.8560\n",
      "Epoch 9/10 | train |  Loss: 0.3409 Acc: 0.8525\n",
      "Epoch 9/10 |  val  |  Loss: 0.3300 Acc: 0.8598\n",
      "Epoch 10/10 | train |  Loss: 0.3312 Acc: 0.8573\n",
      "Epoch 10/10 |  val  |  Loss: 0.3354 Acc: 0.8598\n"
     ]
    }
   ],
   "source": [
    "# 学習・検証を実行する 15分ほどかかります\n",
    "num_epochs = 10\n",
    "net_trained = train_model(net, dataloaders_dict,\n",
    "                          criterion, optimizer, num_epochs=num_epochs)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# テストデータでの正解率を求める"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "テストデータ25000個での正解率：0.8500\n"
     ]
    }
   ],
   "source": [
    "# device\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "net_trained.eval()   # モデルを検証モードに\n",
    "net_trained.to(device)\n",
    "\n",
    "epoch_corrects = 0  # epochの正解数\n",
    "\n",
    "for batch in (test_dl):  # testデータのDataLoader\n",
    "    # batchはTextとLableの辞書オブジェクト\n",
    "    \n",
    "    # GPUが使えるならGPUにデータを送る\n",
    "    inputs = batch.Text[0].to(device)  # 文章\n",
    "    labels = batch.Label.to(device)  # ラベル\n",
    "\n",
    "    # 順伝搬（forward）計算\n",
    "    with torch.set_grad_enabled(False):\n",
    "\n",
    "        # mask作成\n",
    "        input_pad = 1  # 単語のIDにおいて、'<pad>': 1 なので\n",
    "        input_mask = (inputs != input_pad)\n",
    "\n",
    "        # Transformerに入力\n",
    "        outputs, _, _ = net_trained(inputs, input_mask)\n",
    "        _, preds = torch.max(outputs, 1)  # ラベルを予測\n",
    "\n",
    "        # 結果の計算\n",
    "        # 正解数の合計を更新\n",
    "        epoch_corrects += torch.sum(preds == labels.data)\n",
    "\n",
    "# 正解率\n",
    "epoch_acc = epoch_corrects.double() / len(test_dl.dataset)\n",
    "\n",
    "print('テストデータ{}個での正解率：{:.4f}'.format(len(test_dl.dataset),epoch_acc))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Attentionの可視化で判定根拠を探る\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# HTMLを作成する関数を実装\n",
    "\n",
    "\n",
    "def highlight(word, attn):\n",
    "    \"Attentionの値が大きいと文字の背景が濃い赤になるhtmlを出力させる関数\"\n",
    "\n",
    "    html_color = '#%02X%02X%02X' % (\n",
    "        255, int(255*(1 - attn)), int(255*(1 - attn)))\n",
    "    return '<span style=\"background-color: {}\"> {}</span>'.format(html_color, word)\n",
    "\n",
    "\n",
    "def mk_html(index, batch, preds, normlized_weights_1, normlized_weights_2, TEXT):\n",
    "    \"HTMLデータを作成する\"\n",
    "\n",
    "    # indexの結果を抽出\n",
    "    sentence = batch.Text[0][index]  # 文章\n",
    "    label = batch.Label[index]  # ラベル\n",
    "    pred = preds[index]  # 予測\n",
    "\n",
    "    # indexのAttentionを抽出と規格化\n",
    "    attens1 = normlized_weights_1[index, 0, :]  # 0番目の<cls>のAttention\n",
    "    attens1 /= attens1.max()\n",
    "\n",
    "    attens2 = normlized_weights_2[index, 0, :]  # 0番目の<cls>のAttention\n",
    "    attens2 /= attens2.max()\n",
    "\n",
    "    # ラベルと予測結果を文字に置き換え\n",
    "    if label == 0:\n",
    "        label_str = \"Negative\"\n",
    "    else:\n",
    "        label_str = \"Positive\"\n",
    "\n",
    "    if pred == 0:\n",
    "        pred_str = \"Negative\"\n",
    "    else:\n",
    "        pred_str = \"Positive\"\n",
    "\n",
    "    # 表示用のHTMLを作成する\n",
    "    html = '正解ラベル：{}<br>推論ラベル：{}<br><br>'.format(label_str, pred_str)\n",
    "\n",
    "    # 1段目のAttention\n",
    "    html += '[TransformerBlockの1段目のAttentionを可視化]<br>'\n",
    "    for word, attn in zip(sentence, attens1):\n",
    "        html += highlight(TEXT.vocab.itos[word], attn)\n",
    "    html += \"<br><br>\"\n",
    "\n",
    "    # 2段目のAttention\n",
    "    html += '[TransformerBlockの2段目のAttentionを可視化]<br>'\n",
    "    for word, attn in zip(sentence, attens2):\n",
    "        html += highlight(TEXT.vocab.itos[word], attn)\n",
    "\n",
    "    html += \"<br><br>\"\n",
    "\n",
    "    return html\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "正解ラベル：Positive<br>推論ラベル：Positive<br><br>[TransformerBlockの1段目のAttentionを可視化]<br><span style=\"background-color: #FFFCFC\"> <cls></span><span style=\"background-color: #FFFCFC\"> a</span><span style=\"background-color: #FF0000\"> wonderful</span><span style=\"background-color: #FFF7F7\"> film</span><span style=\"background-color: #FFFDFD\"> a</span><span style=\"background-color: #FFDCDC\"> charming</span><span style=\"background-color: #FFFEFE\"> ,</span><span style=\"background-color: #FFCECE\"> quirky</span><span style=\"background-color: #FFFDFD\"> family</span><span style=\"background-color: #FFF8F8\"> story</span><span style=\"background-color: #FFFEFE\"> .</span><span style=\"background-color: #FFFDFD\"> a</span><span style=\"background-color: #FFF9F9\"> cross</span><span style=\"background-color: #FFF7F7\"> country</span><span style=\"background-color: #FFFBFB\"> journey</span><span style=\"background-color: #FFFCFC\"> filled</span><span style=\"background-color: #FFFEFE\"> with</span><span style=\"background-color: #FFFCFC\"> lots</span><span style=\"background-color: #FFFEFE\"> of</span><span style=\"background-color: #FFF4F4\"> interesting</span><span style=\"background-color: #FFFEFE\"> ,</span><span style=\"background-color: #FFF1F1\"> oddball</span><span style=\"background-color: #FFFBFB\"> stops</span><span style=\"background-color: #FFFEFE\"> along</span><span style=\"background-color: #FFFDFD\"> the</span><span style=\"background-color: #FFF6F6\"> way</span><span style=\"background-color: #FFFEFE\"> several</span><span style=\"background-color: #FFFEFE\"> very</span><span style=\"background-color: #FFE7E7\"> cool</span><span style=\"background-color: #FFD9D9\"> cameos</span><span style=\"background-color: #FFFEFE\"> .</span><span style=\"background-color: #FFC7C7\"> great</span><span style=\"background-color: #FFF3F3\"> cast</span><span style=\"background-color: #FFE6E6\"> led</span><span style=\"background-color: #FFFDFD\"> by</span><span style=\"background-color: #FFF7F7\"> rod</span><span style=\"background-color: #FFFDFD\"> <unk></span><span style=\"background-color: #FFFEFE\"> carries</span><span style=\"background-color: #FFFEFE\"> the</span><span style=\"background-color: #FFF9F9\"> film</span><span style=\"background-color: #FFFEFE\"> along</span><span style=\"background-color: #FFFEFE\"> and</span><span style=\"background-color: #FFEDED\"> leads</span><span style=\"background-color: #FFFEFE\"> to</span><span style=\"background-color: #FFFDFD\"> a</span><span style=\"background-color: #FFFAFA\"> surprise</span><span style=\"background-color: #FFFEFE\"> ending</span><span style=\"background-color: #FFFEFE\"> .</span><span style=\"background-color: #FFFDFD\"> well</span><span style=\"background-color: #FFECEC\"> directed</span><span style=\"background-color: #FFFDFD\"> shot</span><span style=\"background-color: #FFFDFD\"> a</span><span style=\"background-color: #FFF5F5\"> really</span><span style=\"background-color: #FFD3D3\"> nice</span><span style=\"background-color: #FFF4F4\"> movie</span><span style=\"background-color: #FFFEFE\"> .</span><span style=\"background-color: #FFFEFE\"> <eos></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><br><br>[TransformerBlockの2段目のAttentionを可視化]<br><span style=\"background-color: #FFFEFE\"> <cls></span><span style=\"background-color: #FFFAFA\"> a</span><span style=\"background-color: #FFB9B9\"> wonderful</span><span style=\"background-color: #FFFCFC\"> film</span><span style=\"background-color: #FFF4F4\"> a</span><span style=\"background-color: #FF9595\"> charming</span><span style=\"background-color: #FFF5F5\"> ,</span><span style=\"background-color: #FFF8F8\"> quirky</span><span style=\"background-color: #FFFEFE\"> family</span><span style=\"background-color: #FFFEFE\"> story</span><span style=\"background-color: #FFF4F4\"> .</span><span style=\"background-color: #FFF9F9\"> a</span><span style=\"background-color: #FFF8F8\"> cross</span><span style=\"background-color: #FFFAFA\"> country</span><span style=\"background-color: #FFFEFE\"> journey</span><span style=\"background-color: #FFEBEB\"> filled</span><span style=\"background-color: #FFE3E3\"> with</span><span style=\"background-color: #FFF2F2\"> lots</span><span style=\"background-color: #FFF7F7\"> of</span><span style=\"background-color: #FFF2F2\"> interesting</span><span style=\"background-color: #FFF9F9\"> ,</span><span style=\"background-color: #FFFBFB\"> oddball</span><span style=\"background-color: #FFF5F5\"> stops</span><span style=\"background-color: #FFDBDB\"> along</span><span style=\"background-color: #FFFDFD\"> the</span><span style=\"background-color: #FFD8D8\"> way</span><span style=\"background-color: #FFCFCF\"> several</span><span style=\"background-color: #FF6262\"> very</span><span style=\"background-color: #FF9292\"> cool</span><span style=\"background-color: #FFFDFD\"> cameos</span><span style=\"background-color: #FFE9E9\"> .</span><span style=\"background-color: #FF0000\"> great</span><span style=\"background-color: #FFFCFC\"> cast</span><span style=\"background-color: #FFFEFE\"> led</span><span style=\"background-color: #FFF6F6\"> by</span><span style=\"background-color: #FFF9F9\"> rod</span><span style=\"background-color: #FFFCFC\"> <unk></span><span style=\"background-color: #FFF5F5\"> carries</span><span style=\"background-color: #FFFEFE\"> the</span><span style=\"background-color: #FFFDFD\"> film</span><span style=\"background-color: #FFF1F1\"> along</span><span style=\"background-color: #FFFBFB\"> and</span><span style=\"background-color: #FFFCFC\"> leads</span><span style=\"background-color: #FFFDFD\"> to</span><span style=\"background-color: #FFF8F8\"> a</span><span style=\"background-color: #FFFBFB\"> surprise</span><span style=\"background-color: #FFFDFD\"> ending</span><span style=\"background-color: #FFF5F5\"> .</span><span style=\"background-color: #FF9B9B\"> well</span><span style=\"background-color: #FFF7F7\"> directed</span><span style=\"background-color: #FFFBFB\"> shot</span><span style=\"background-color: #FFF8F8\"> a</span><span style=\"background-color: #FFFCFC\"> really</span><span style=\"background-color: #FFA6A6\"> nice</span><span style=\"background-color: #FFFEFE\"> movie</span><span style=\"background-color: #FFF0F0\"> .</span><span style=\"background-color: #FFFEFE\"> <eos></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import HTML\n",
    "\n",
    "# Transformerで処理\n",
    "\n",
    "# ミニバッチの用意\n",
    "batch = next(iter(test_dl))\n",
    "\n",
    "# GPUが使えるならGPUにデータを送る\n",
    "inputs = batch.Text[0].to(device)  # 文章\n",
    "labels = batch.Label.to(device)  # ラベル\n",
    "\n",
    "# mask作成\n",
    "input_pad = 1  # 単語のIDにおいて、'<pad>': 1 なので\n",
    "input_mask = (inputs != input_pad)\n",
    "\n",
    "# Transformerに入力\n",
    "outputs, normlized_weights_1, normlized_weights_2 = net_trained(\n",
    "    inputs, input_mask)\n",
    "_, preds = torch.max(outputs, 1)  # ラベルを予測\n",
    "\n",
    "\n",
    "index = 3  # 出力させたいデータ\n",
    "html_output = mk_html(index, batch, preds, normlized_weights_1,\n",
    "                      normlized_weights_2, TEXT)  # HTML作成\n",
    "HTML(html_output)  # HTML形式で出力\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "正解ラベル：Positive<br>推論ラベル：Negative<br><br>[TransformerBlockの1段目のAttentionを可視化]<br><span style=\"background-color: #FFF7F7\"> <cls></span><span style=\"background-color: #FFFAFA\"> i</span><span style=\"background-color: #FFD5D5\"> watched</span><span style=\"background-color: #FFF0F0\"> this</span><span style=\"background-color: #FFD4D4\"> movie</span><span style=\"background-color: #FFFEFE\"> as</span><span style=\"background-color: #FFFBFB\"> a</span><span style=\"background-color: #FFFEFE\"> child</span><span style=\"background-color: #FFFEFE\"> and</span><span style=\"background-color: #FFF5F5\"> still</span><span style=\"background-color: #FF7C7C\"> enjoy</span><span style=\"background-color: #FFCDCD\"> viewing</span><span style=\"background-color: #FFFDFD\"> it</span><span style=\"background-color: #FFFDFD\"> every</span><span style=\"background-color: #FFFCFC\"> once</span><span style=\"background-color: #FFFEFE\"> in</span><span style=\"background-color: #FFFBFB\"> a</span><span style=\"background-color: #FFFEFE\"> while</span><span style=\"background-color: #FFFEFE\"> for</span><span style=\"background-color: #FFFDFD\"> the</span><span style=\"background-color: #FFDBDB\"> nostalgia</span><span style=\"background-color: #FFF8F8\"> factor</span><span style=\"background-color: #FFFEFE\"> .</span><span style=\"background-color: #FFFEFE\"> when</span><span style=\"background-color: #FFFBFB\"> i</span><span style=\"background-color: #FFE8E8\"> was</span><span style=\"background-color: #FFF5F5\"> younger</span><span style=\"background-color: #FFFBFB\"> i</span><span style=\"background-color: #FF0000\"> loved</span><span style=\"background-color: #FFFCFC\"> the</span><span style=\"background-color: #FFD3D3\"> movie</span><span style=\"background-color: #FFFCFC\"> because</span><span style=\"background-color: #FFFCFC\"> of</span><span style=\"background-color: #FFFCFC\"> the</span><span style=\"background-color: #FF7272\"> entertaining</span><span style=\"background-color: #FFCBCB\"> storyline</span><span style=\"background-color: #FFFEFE\"> and</span><span style=\"background-color: #FFCFCF\"> interesting</span><span style=\"background-color: #FFDEDE\"> characters</span><span style=\"background-color: #FFFEFE\"> .</span><span style=\"background-color: #FFFBFB\"> today</span><span style=\"background-color: #FFFEFE\"> ,</span><span style=\"background-color: #FFFBFB\"> i</span><span style=\"background-color: #FFF6F6\"> still</span><span style=\"background-color: #FFB5B5\"> love</span><span style=\"background-color: #FFFCFC\"> the</span><span style=\"background-color: #FFE0E0\"> characters</span><span style=\"background-color: #FFFEFE\"> .</span><span style=\"background-color: #FFFCFC\"> additionally</span><span style=\"background-color: #FFFEFE\"> ,</span><span style=\"background-color: #FFFBFB\"> i</span><span style=\"background-color: #FFF8F8\"> think</span><span style=\"background-color: #FFFBFB\"> of</span><span style=\"background-color: #FFFDFD\"> the</span><span style=\"background-color: #FFC5C5\"> plot</span><span style=\"background-color: #FFFEFE\"> with</span><span style=\"background-color: #FFFCFC\"> higher</span><span style=\"background-color: #FFD4D4\"> regard</span><span style=\"background-color: #FFFBFB\"> because</span><span style=\"background-color: #FFFBFB\"> i</span><span style=\"background-color: #FFFDFD\"> now</span><span style=\"background-color: #FFF7F7\"> see</span><span style=\"background-color: #FFFCFC\"> the</span><span style=\"background-color: #FFC9C9\"> morals</span><span style=\"background-color: #FFFEFE\"> and</span><span style=\"background-color: #FFF3F3\"> symbolism</span><span style=\"background-color: #FFFEFE\"> .</span><span style=\"background-color: #FFE4E4\"> rainbow</span><span style=\"background-color: #FFFCFC\"> <unk></span><span style=\"background-color: #FFFBFB\"> is</span><span style=\"background-color: #FFFCFC\"> far</span><span style=\"background-color: #FFFEFE\"> from</span><span style=\"background-color: #FFFCFC\"> the</span><span style=\"background-color: #FFB8B8\"> worst</span><span style=\"background-color: #FFE2E2\"> film</span><span style=\"background-color: #FFADAD\"> ever</span><span style=\"background-color: #FFFEFE\"> ,</span><span style=\"background-color: #FFFDFD\"> and</span><span style=\"background-color: #FFFAFA\"> though</span><span style=\"background-color: #FFFDFD\"> out</span><span style=\"background-color: #FFF3F3\"> dated</span><span style=\"background-color: #FFFEFE\"> ,</span><span style=\"background-color: #FFFBFB\"> i</span><span style=\"background-color: #FFFDFD\"> m</span><span style=\"background-color: #FFEBEB\"> sure</span><span style=\"background-color: #FFFAFA\"> i</span><span style=\"background-color: #FFEBEB\"> will</span><span style=\"background-color: #FFF6F6\"> show</span><span style=\"background-color: #FFFDFD\"> it</span><span style=\"background-color: #FFFCFC\"> to</span><span style=\"background-color: #FFF7F7\"> my</span><span style=\"background-color: #FFFCFC\"> children</span><span style=\"background-color: #FFFEFE\"> in</span><span style=\"background-color: #FFFCFC\"> the</span><span style=\"background-color: #FFF8F8\"> future</span><span style=\"background-color: #FFFEFE\"> ,</span><span style=\"background-color: #FFFEFE\"> when</span><span style=\"background-color: #FFFAFA\"> i</span><span style=\"background-color: #FFF3F3\"> have</span><span style=\"background-color: #FFFAFA\"> children</span><span style=\"background-color: #FFFEFE\"> .</span><span style=\"background-color: #FFFBFB\"> <eos></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><br><br>[TransformerBlockの2段目のAttentionを可視化]<br><span style=\"background-color: #FFFEFE\"> <cls></span><span style=\"background-color: #FFFEFE\"> i</span><span style=\"background-color: #FFFEFE\"> watched</span><span style=\"background-color: #FFFDFD\"> this</span><span style=\"background-color: #FFFEFE\"> movie</span><span style=\"background-color: #FFFEFE\"> as</span><span style=\"background-color: #FFFEFE\"> a</span><span style=\"background-color: #FFFEFE\"> child</span><span style=\"background-color: #FFFEFE\"> and</span><span style=\"background-color: #FFFEFE\"> still</span><span style=\"background-color: #FFFCFC\"> enjoy</span><span style=\"background-color: #FFFEFE\"> viewing</span><span style=\"background-color: #FFFEFE\"> it</span><span style=\"background-color: #FFFEFE\"> every</span><span style=\"background-color: #FFFEFE\"> once</span><span style=\"background-color: #FFFEFE\"> in</span><span style=\"background-color: #FFFEFE\"> a</span><span style=\"background-color: #FFFDFD\"> while</span><span style=\"background-color: #FFFEFE\"> for</span><span style=\"background-color: #FFFEFE\"> the</span><span style=\"background-color: #FFFDFD\"> nostalgia</span><span style=\"background-color: #FFFEFE\"> factor</span><span style=\"background-color: #FFFEFE\"> .</span><span style=\"background-color: #FFFEFE\"> when</span><span style=\"background-color: #FFFEFE\"> i</span><span style=\"background-color: #FFFEFE\"> was</span><span style=\"background-color: #FFFEFE\"> younger</span><span style=\"background-color: #FFFEFE\"> i</span><span style=\"background-color: #FFFDFD\"> loved</span><span style=\"background-color: #FFFEFE\"> the</span><span style=\"background-color: #FFFEFE\"> movie</span><span style=\"background-color: #FFFEFE\"> because</span><span style=\"background-color: #FFFEFE\"> of</span><span style=\"background-color: #FFFEFE\"> the</span><span style=\"background-color: #FFF7F7\"> entertaining</span><span style=\"background-color: #FFFEFE\"> storyline</span><span style=\"background-color: #FFFEFE\"> and</span><span style=\"background-color: #FFFDFD\"> interesting</span><span style=\"background-color: #FFFEFE\"> characters</span><span style=\"background-color: #FFFEFE\"> .</span><span style=\"background-color: #FFFEFE\"> today</span><span style=\"background-color: #FFFEFE\"> ,</span><span style=\"background-color: #FFFEFE\"> i</span><span style=\"background-color: #FFFEFE\"> still</span><span style=\"background-color: #FFFEFE\"> love</span><span style=\"background-color: #FFFEFE\"> the</span><span style=\"background-color: #FFFEFE\"> characters</span><span style=\"background-color: #FFFEFE\"> .</span><span style=\"background-color: #FFFCFC\"> additionally</span><span style=\"background-color: #FFFEFE\"> ,</span><span style=\"background-color: #FFFEFE\"> i</span><span style=\"background-color: #FFFEFE\"> think</span><span style=\"background-color: #FFFEFE\"> of</span><span style=\"background-color: #FFFEFE\"> the</span><span style=\"background-color: #FFFEFE\"> plot</span><span style=\"background-color: #FFFEFE\"> with</span><span style=\"background-color: #FFF6F6\"> higher</span><span style=\"background-color: #FFFEFE\"> regard</span><span style=\"background-color: #FFFEFE\"> because</span><span style=\"background-color: #FFFEFE\"> i</span><span style=\"background-color: #FFFEFE\"> now</span><span style=\"background-color: #FFFEFE\"> see</span><span style=\"background-color: #FFFEFE\"> the</span><span style=\"background-color: #FFFEFE\"> morals</span><span style=\"background-color: #FFFEFE\"> and</span><span style=\"background-color: #FFFDFD\"> symbolism</span><span style=\"background-color: #FFFEFE\"> .</span><span style=\"background-color: #FFFEFE\"> rainbow</span><span style=\"background-color: #FFFEFE\"> <unk></span><span style=\"background-color: #FFFDFD\"> is</span><span style=\"background-color: #FFFDFD\"> far</span><span style=\"background-color: #FFFEFE\"> from</span><span style=\"background-color: #FFFEFE\"> the</span><span style=\"background-color: #FF0000\"> worst</span><span style=\"background-color: #FFFEFE\"> film</span><span style=\"background-color: #FFFEFE\"> ever</span><span style=\"background-color: #FFFEFE\"> ,</span><span style=\"background-color: #FFFEFE\"> and</span><span style=\"background-color: #FFFDFD\"> though</span><span style=\"background-color: #FFFCFC\"> out</span><span style=\"background-color: #FFFEFE\"> dated</span><span style=\"background-color: #FFFEFE\"> ,</span><span style=\"background-color: #FFFEFE\"> i</span><span style=\"background-color: #FFFEFE\"> m</span><span style=\"background-color: #FFF9F9\"> sure</span><span style=\"background-color: #FFFEFE\"> i</span><span style=\"background-color: #FFFEFE\"> will</span><span style=\"background-color: #FFFEFE\"> show</span><span style=\"background-color: #FFFEFE\"> it</span><span style=\"background-color: #FFFEFE\"> to</span><span style=\"background-color: #FFFEFE\"> my</span><span style=\"background-color: #FFFEFE\"> children</span><span style=\"background-color: #FFFEFE\"> in</span><span style=\"background-color: #FFFEFE\"> the</span><span style=\"background-color: #FFFCFC\"> future</span><span style=\"background-color: #FFFEFE\"> ,</span><span style=\"background-color: #FFFEFE\"> when</span><span style=\"background-color: #FFFEFE\"> i</span><span style=\"background-color: #FFFEFE\"> have</span><span style=\"background-color: #FFFEFE\"> children</span><span style=\"background-color: #FFFEFE\"> .</span><span style=\"background-color: #FFFEFE\"> <eos></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><span style=\"background-color: #FFFFFF\"> <pad></span><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index = 61  # 出力させたいデータ\n",
    "html_output = mk_html(index, batch, preds, normlized_weights_1,\n",
    "                      normlized_weights_2, TEXT)  # HTML作成\n",
    "HTML(html_output)  # HTML形式で出力\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "以上"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
